{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('iteration', 0, ':cross-entropy-loss', array(545.5765380859375, dtype=float32))\n",
      "('iteration', 100, ':cross-entropy-loss', array(99.25707244873047, dtype=float32))\n",
      "('iteration', 200, ':cross-entropy-loss', array(81.77348327636719, dtype=float32))\n",
      "('iteration', 300, ':cross-entropy-loss', array(76.42056274414062, dtype=float32))\n",
      "('iteration', 400, ':cross-entropy-loss', array(67.81326293945312, dtype=float32))\n",
      "('iteration', 500, ':cross-entropy-loss', array(64.7454605102539, dtype=float32))\n",
      "('iteration', 600, ':cross-entropy-loss', array(71.48340606689453, dtype=float32))\n",
      "('iteration', 700, ':cross-entropy-loss', array(66.0996322631836, dtype=float32))\n",
      "('iteration', 800, ':cross-entropy-loss', array(66.06291198730469, dtype=float32))\n",
      "('iteration', 900, ':cross-entropy-loss', array(68.5797348022461, dtype=float32))\n",
      "('iteration', 1000, ':cross-entropy-loss', array(62.84904861450195, dtype=float32))\n",
      "('iteration', 1100, ':cross-entropy-loss', array(59.88761901855469, dtype=float32))\n",
      "('iteration', 1200, ':cross-entropy-loss', array(65.69959259033203, dtype=float32))\n",
      "('iteration', 1300, ':cross-entropy-loss', array(63.8610725402832, dtype=float32))\n",
      "('iteration', 1400, ':cross-entropy-loss', array(63.6240348815918, dtype=float32))\n",
      "('iteration', 1500, ':cross-entropy-loss', array(66.24593353271484, dtype=float32))\n",
      "('iteration', 1600, ':cross-entropy-loss', array(60.897762298583984, dtype=float32))\n",
      "('iteration', 1700, ':cross-entropy-loss', array(58.155887603759766, dtype=float32))\n",
      "('iteration', 1800, ':cross-entropy-loss', array(62.271053314208984, dtype=float32))\n",
      "('iteration', 1900, ':cross-entropy-loss', array(62.40495681762695, dtype=float32))\n",
      "('iteration', 2000, ':cross-entropy-loss', array(62.91985321044922, dtype=float32))\n",
      "('iteration', 2100, ':cross-entropy-loss', array(65.37183380126953, dtype=float32))\n",
      "('iteration', 2200, ':cross-entropy-loss', array(59.651180267333984, dtype=float32))\n",
      "('iteration', 2300, ':cross-entropy-loss', array(57.06944274902344, dtype=float32))\n",
      "('iteration', 2400, ':cross-entropy-loss', array(60.463417053222656, dtype=float32))\n",
      "('iteration', 2500, ':cross-entropy-loss', array(61.26860427856445, dtype=float32))\n",
      "('iteration', 2600, ':cross-entropy-loss', array(61.85260772705078, dtype=float32))\n",
      "('iteration', 2700, ':cross-entropy-loss', array(63.774314880371094, dtype=float32))\n",
      "('iteration', 2800, ':cross-entropy-loss', array(59.209754943847656, dtype=float32))\n",
      "('iteration', 2900, ':cross-entropy-loss', array(56.635284423828125, dtype=float32))\n",
      "('iteration', 3000, ':cross-entropy-loss', array(60.324466705322266, dtype=float32))\n",
      "('iteration', 3100, ':cross-entropy-loss', array(60.7249755859375, dtype=float32))\n",
      "('iteration', 3200, ':cross-entropy-loss', array(61.7202262878418, dtype=float32))\n",
      "('iteration', 3300, ':cross-entropy-loss', array(63.483009338378906, dtype=float32))\n",
      "('iteration', 3400, ':cross-entropy-loss', array(58.95928192138672, dtype=float32))\n",
      "('iteration', 3500, ':cross-entropy-loss', array(56.87931823730469, dtype=float32))\n",
      "('iteration', 3600, ':cross-entropy-loss', array(59.8358268737793, dtype=float32))\n",
      "('iteration', 3700, ':cross-entropy-loss', array(60.61467361450195, dtype=float32))\n",
      "('iteration', 3800, ':cross-entropy-loss', array(61.627140045166016, dtype=float32))\n",
      "('iteration', 3900, ':cross-entropy-loss', array(63.34425735473633, dtype=float32))\n",
      "('iteration', 4000, ':cross-entropy-loss', array(59.213584899902344, dtype=float32))\n",
      "('iteration', 4100, ':cross-entropy-loss', array(56.75306701660156, dtype=float32))\n",
      "('iteration', 4200, ':cross-entropy-loss', array(59.743282318115234, dtype=float32))\n",
      "('iteration', 4300, ':cross-entropy-loss', array(60.69478988647461, dtype=float32))\n",
      "('iteration', 4400, ':cross-entropy-loss', array(61.5949592590332, dtype=float32))\n",
      "('iteration', 4500, ':cross-entropy-loss', array(63.0605354309082, dtype=float32))\n",
      "('iteration', 4600, ':cross-entropy-loss', array(58.872745513916016, dtype=float32))\n",
      "('iteration', 4700, ':cross-entropy-loss', array(56.84634780883789, dtype=float32))\n",
      "('iteration', 4800, ':cross-entropy-loss', array(59.76994323730469, dtype=float32))\n",
      "('iteration', 4900, ':cross-entropy-loss', array(60.932891845703125, dtype=float32))\n",
      "('iteration', 5000, ':cross-entropy-loss', array(61.21330261230469, dtype=float32))\n",
      "('iteration', 5100, ':cross-entropy-loss', array(62.90462875366211, dtype=float32))\n",
      "('iteration', 5200, ':cross-entropy-loss', array(58.5720329284668, dtype=float32))\n",
      "('iteration', 5300, ':cross-entropy-loss', array(56.87971496582031, dtype=float32))\n",
      "('iteration', 5400, ':cross-entropy-loss', array(60.10886764526367, dtype=float32))\n",
      "('iteration', 5500, ':cross-entropy-loss', array(61.048377990722656, dtype=float32))\n",
      "('iteration', 5600, ':cross-entropy-loss', array(61.45631408691406, dtype=float32))\n",
      "('iteration', 5700, ':cross-entropy-loss', array(62.969451904296875, dtype=float32))\n",
      "('iteration', 5800, ':cross-entropy-loss', array(59.23566818237305, dtype=float32))\n",
      "('iteration', 5900, ':cross-entropy-loss', array(57.121116638183594, dtype=float32))\n",
      "('iteration', 6000, ':cross-entropy-loss', array(60.8026237487793, dtype=float32))\n",
      "('iteration', 6100, ':cross-entropy-loss', array(61.220848083496094, dtype=float32))\n",
      "('iteration', 6200, ':cross-entropy-loss', array(61.61957550048828, dtype=float32))\n",
      "('iteration', 6300, ':cross-entropy-loss', array(63.41067886352539, dtype=float32))\n",
      "('iteration', 6400, ':cross-entropy-loss', array(59.75441360473633, dtype=float32))\n",
      "('iteration', 6500, ':cross-entropy-loss', array(57.31150436401367, dtype=float32))\n",
      "('iteration', 6600, ':cross-entropy-loss', array(60.7581901550293, dtype=float32))\n",
      "('iteration', 6700, ':cross-entropy-loss', array(61.69132995605469, dtype=float32))\n",
      "('iteration', 6800, ':cross-entropy-loss', array(61.717674255371094, dtype=float32))\n",
      "('iteration', 6900, ':cross-entropy-loss', array(63.56916046142578, dtype=float32))\n",
      "('iteration', 7000, ':cross-entropy-loss', array(59.88785171508789, dtype=float32))\n",
      "('iteration', 7100, ':cross-entropy-loss', array(57.37657165527344, dtype=float32))\n",
      "('iteration', 7200, ':cross-entropy-loss', array(60.615447998046875, dtype=float32))\n",
      "('iteration', 7300, ':cross-entropy-loss', array(61.86330032348633, dtype=float32))\n",
      "('iteration', 7400, ':cross-entropy-loss', array(62.472801208496094, dtype=float32))\n",
      "('iteration', 7500, ':cross-entropy-loss', array(63.908077239990234, dtype=float32))\n",
      "('iteration', 7600, ':cross-entropy-loss', array(60.103858947753906, dtype=float32))\n",
      "('iteration', 7700, ':cross-entropy-loss', array(57.78468704223633, dtype=float32))\n",
      "('iteration', 7800, ':cross-entropy-loss', array(60.73139953613281, dtype=float32))\n",
      "('iteration', 7900, ':cross-entropy-loss', array(62.01778793334961, dtype=float32))\n",
      "('iteration', 8000, ':cross-entropy-loss', array(62.43880844116211, dtype=float32))\n",
      "('iteration', 8100, ':cross-entropy-loss', array(64.0378189086914, dtype=float32))\n",
      "('iteration', 8200, ':cross-entropy-loss', array(60.33245086669922, dtype=float32))\n",
      "('iteration', 8300, ':cross-entropy-loss', array(58.108734130859375, dtype=float32))\n",
      "('iteration', 8400, ':cross-entropy-loss', array(60.607383728027344, dtype=float32))\n",
      "('iteration', 8500, ':cross-entropy-loss', array(62.28081130981445, dtype=float32))\n",
      "('iteration', 8600, ':cross-entropy-loss', array(62.69929504394531, dtype=float32))\n",
      "('iteration', 8700, ':cross-entropy-loss', array(63.869022369384766, dtype=float32))\n",
      "('iteration', 8800, ':cross-entropy-loss', array(60.290000915527344, dtype=float32))\n",
      "('iteration', 8900, ':cross-entropy-loss', array(58.33009719848633, dtype=float32))\n",
      "('iteration', 9000, ':cross-entropy-loss', array(60.85411071777344, dtype=float32))\n",
      "('iteration', 9100, ':cross-entropy-loss', array(61.73069381713867, dtype=float32))\n",
      "('iteration', 9200, ':cross-entropy-loss', array(61.99649429321289, dtype=float32))\n",
      "('iteration', 9300, ':cross-entropy-loss', array(63.992027282714844, dtype=float32))\n",
      "('iteration', 9400, ':cross-entropy-loss', array(60.274314880371094, dtype=float32))\n",
      "('iteration', 9500, ':cross-entropy-loss', array(58.05473709106445, dtype=float32))\n",
      "('iteration', 9600, ':cross-entropy-loss', array(60.80146408081055, dtype=float32))\n",
      "('iteration', 9700, ':cross-entropy-loss', array(61.886837005615234, dtype=float32))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('iteration', 9800, ':cross-entropy-loss', array(62.18116760253906, dtype=float32))\n",
      "('iteration', 9900, ':cross-entropy-loss', array(64.01029968261719, dtype=float32))\n",
      "('iteration', 10000, ':cross-entropy-loss', array(60.244239807128906, dtype=float32))\n",
      "('iteration', 10100, ':cross-entropy-loss', array(57.934513092041016, dtype=float32))\n",
      "('iteration', 10200, ':cross-entropy-loss', array(60.89118957519531, dtype=float32))\n",
      "('iteration', 10300, ':cross-entropy-loss', array(61.873409271240234, dtype=float32))\n",
      "('iteration', 10400, ':cross-entropy-loss', array(62.70445251464844, dtype=float32))\n",
      "('iteration', 10500, ':cross-entropy-loss', array(64.2917709350586, dtype=float32))\n",
      "('iteration', 10600, ':cross-entropy-loss', array(60.168357849121094, dtype=float32))\n",
      "('iteration', 10700, ':cross-entropy-loss', array(58.04652404785156, dtype=float32))\n",
      "('iteration', 10800, ':cross-entropy-loss', array(61.1159553527832, dtype=float32))\n",
      "('iteration', 10900, ':cross-entropy-loss', array(62.30500030517578, dtype=float32))\n",
      "('iteration', 11000, ':cross-entropy-loss', array(63.32963943481445, dtype=float32))\n",
      "('iteration', 11100, ':cross-entropy-loss', array(64.63232421875, dtype=float32))\n",
      "('iteration', 11200, ':cross-entropy-loss', array(60.628868103027344, dtype=float32))\n",
      "('iteration', 11300, ':cross-entropy-loss', array(58.53388595581055, dtype=float32))\n",
      "('iteration', 11400, ':cross-entropy-loss', array(61.18977355957031, dtype=float32))\n",
      "('iteration', 11500, ':cross-entropy-loss', array(62.446746826171875, dtype=float32))\n",
      "('iteration', 11600, ':cross-entropy-loss', array(63.440982818603516, dtype=float32))\n",
      "('iteration', 11700, ':cross-entropy-loss', array(65.2787857055664, dtype=float32))\n",
      "('iteration', 11800, ':cross-entropy-loss', array(61.075653076171875, dtype=float32))\n",
      "('iteration', 11900, ':cross-entropy-loss', array(58.764957427978516, dtype=float32))\n",
      "('iteration', 12000, ':cross-entropy-loss', array(60.93476486206055, dtype=float32))\n",
      "('iteration', 12100, ':cross-entropy-loss', array(62.39039993286133, dtype=float32))\n",
      "('iteration', 12200, ':cross-entropy-loss', array(63.963233947753906, dtype=float32))\n",
      "('iteration', 12300, ':cross-entropy-loss', array(65.12971496582031, dtype=float32))\n",
      "('iteration', 12400, ':cross-entropy-loss', array(61.439456939697266, dtype=float32))\n",
      "('iteration', 12500, ':cross-entropy-loss', array(59.58781433105469, dtype=float32))\n",
      "('iteration', 12600, ':cross-entropy-loss', array(61.296573638916016, dtype=float32))\n",
      "('iteration', 12700, ':cross-entropy-loss', array(62.927284240722656, dtype=float32))\n",
      "('iteration', 12800, ':cross-entropy-loss', array(63.7689094543457, dtype=float32))\n",
      "('iteration', 12900, ':cross-entropy-loss', array(65.40723419189453, dtype=float32))\n",
      "('iteration', 13000, ':cross-entropy-loss', array(61.44937515258789, dtype=float32))\n",
      "('iteration', 13100, ':cross-entropy-loss', array(59.51252746582031, dtype=float32))\n",
      "('iteration', 13200, ':cross-entropy-loss', array(61.63761901855469, dtype=float32))\n",
      "('iteration', 13300, ':cross-entropy-loss', array(62.962947845458984, dtype=float32))\n",
      "('iteration', 13400, ':cross-entropy-loss', array(63.63059616088867, dtype=float32))\n",
      "('iteration', 13500, ':cross-entropy-loss', array(65.27649688720703, dtype=float32))\n",
      "('iteration', 13600, ':cross-entropy-loss', array(61.314754486083984, dtype=float32))\n",
      "('iteration', 13700, ':cross-entropy-loss', array(59.796241760253906, dtype=float32))\n",
      "('iteration', 13800, ':cross-entropy-loss', array(62.056209564208984, dtype=float32))\n",
      "('iteration', 13900, ':cross-entropy-loss', array(63.236488342285156, dtype=float32))\n",
      "('iteration', 14000, ':cross-entropy-loss', array(64.08273315429688, dtype=float32))\n",
      "('iteration', 14100, ':cross-entropy-loss', array(66.12056732177734, dtype=float32))\n",
      "('iteration', 14200, ':cross-entropy-loss', array(62.059513092041016, dtype=float32))\n",
      "('iteration', 14300, ':cross-entropy-loss', array(60.336952209472656, dtype=float32))\n",
      "('iteration', 14400, ':cross-entropy-loss', array(62.360565185546875, dtype=float32))\n",
      "('iteration', 14500, ':cross-entropy-loss', array(63.260650634765625, dtype=float32))\n",
      "('iteration', 14600, ':cross-entropy-loss', array(64.55613708496094, dtype=float32))\n",
      "('iteration', 14700, ':cross-entropy-loss', array(65.92172241210938, dtype=float32))\n",
      "('iteration', 14800, ':cross-entropy-loss', array(62.501182556152344, dtype=float32))\n",
      "('iteration', 14900, ':cross-entropy-loss', array(60.743045806884766, dtype=float32))\n",
      "('iteration', 15000, ':cross-entropy-loss', array(62.63420867919922, dtype=float32))\n",
      "('iteration', 15100, ':cross-entropy-loss', array(63.46573257446289, dtype=float32))\n",
      "('iteration', 15200, ':cross-entropy-loss', array(64.67119598388672, dtype=float32))\n",
      "('iteration', 15300, ':cross-entropy-loss', array(66.63982391357422, dtype=float32))\n",
      "('iteration', 15400, ':cross-entropy-loss', array(62.5493049621582, dtype=float32))\n",
      "('iteration', 15500, ':cross-entropy-loss', array(60.565181732177734, dtype=float32))\n",
      "('iteration', 15600, ':cross-entropy-loss', array(62.857460021972656, dtype=float32))\n",
      "('iteration', 15700, ':cross-entropy-loss', array(63.59883117675781, dtype=float32))\n",
      "('iteration', 15800, ':cross-entropy-loss', array(64.43984985351562, dtype=float32))\n",
      "('iteration', 15900, ':cross-entropy-loss', array(66.79265594482422, dtype=float32))\n",
      "('iteration', 16000, ':cross-entropy-loss', array(62.39775466918945, dtype=float32))\n",
      "('iteration', 16100, ':cross-entropy-loss', array(60.40028381347656, dtype=float32))\n",
      "('iteration', 16200, ':cross-entropy-loss', array(62.795928955078125, dtype=float32))\n",
      "('iteration', 16300, ':cross-entropy-loss', array(63.887237548828125, dtype=float32))\n",
      "('iteration', 16400, ':cross-entropy-loss', array(64.77654266357422, dtype=float32))\n",
      "('iteration', 16500, ':cross-entropy-loss', array(66.9535140991211, dtype=float32))\n",
      "('iteration', 16600, ':cross-entropy-loss', array(62.99778366088867, dtype=float32))\n",
      "('iteration', 16700, ':cross-entropy-loss', array(60.457679748535156, dtype=float32))\n",
      "('iteration', 16800, ':cross-entropy-loss', array(63.007720947265625, dtype=float32))\n",
      "('iteration', 16900, ':cross-entropy-loss', array(64.21900939941406, dtype=float32))\n",
      "('iteration', 17000, ':cross-entropy-loss', array(65.3892593383789, dtype=float32))\n",
      "('iteration', 17100, ':cross-entropy-loss', array(67.15209197998047, dtype=float32))\n",
      "('iteration', 17200, ':cross-entropy-loss', array(62.94367218017578, dtype=float32))\n",
      "('iteration', 17300, ':cross-entropy-loss', array(60.238887786865234, dtype=float32))\n",
      "('iteration', 17400, ':cross-entropy-loss', array(63.38680648803711, dtype=float32))\n",
      "('iteration', 17500, ':cross-entropy-loss', array(64.33231353759766, dtype=float32))\n",
      "('iteration', 17600, ':cross-entropy-loss', array(65.42304992675781, dtype=float32))\n",
      "('iteration', 17700, ':cross-entropy-loss', array(67.71160888671875, dtype=float32))\n",
      "('iteration', 17800, ':cross-entropy-loss', array(63.311309814453125, dtype=float32))\n",
      "('iteration', 17900, ':cross-entropy-loss', array(60.661556243896484, dtype=float32))\n",
      "('iteration', 18000, ':cross-entropy-loss', array(63.2187385559082, dtype=float32))\n",
      "('iteration', 18100, ':cross-entropy-loss', array(64.51741027832031, dtype=float32))\n",
      "('iteration', 18200, ':cross-entropy-loss', array(66.14776611328125, dtype=float32))\n",
      "('iteration', 18300, ':cross-entropy-loss', array(67.48147583007812, dtype=float32))\n",
      "('iteration', 18400, ':cross-entropy-loss', array(63.26853561401367, dtype=float32))\n",
      "('iteration', 18500, ':cross-entropy-loss', array(61.16401290893555, dtype=float32))\n",
      "('iteration', 18600, ':cross-entropy-loss', array(63.70392990112305, dtype=float32))\n",
      "('iteration', 18700, ':cross-entropy-loss', array(64.92399597167969, dtype=float32))\n",
      "('iteration', 18800, ':cross-entropy-loss', array(66.09542846679688, dtype=float32))\n",
      "('iteration', 18900, ':cross-entropy-loss', array(67.76680755615234, dtype=float32))\n",
      "('iteration', 19000, ':cross-entropy-loss', array(63.71323776245117, dtype=float32))\n",
      "('iteration', 19100, ':cross-entropy-loss', array(61.243499755859375, dtype=float32))\n",
      "('iteration', 19200, ':cross-entropy-loss', array(64.17829132080078, dtype=float32))\n",
      "('iteration', 19300, ':cross-entropy-loss', array(65.06832122802734, dtype=float32))\n",
      "('iteration', 19400, ':cross-entropy-loss', array(66.09024047851562, dtype=float32))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('iteration', 19500, ':cross-entropy-loss', array(68.66814422607422, dtype=float32))\n",
      "('iteration', 19600, ':cross-entropy-loss', array(63.96269226074219, dtype=float32))\n",
      "('iteration', 19700, ':cross-entropy-loss', array(61.38185501098633, dtype=float32))\n",
      "('iteration', 19800, ':cross-entropy-loss', array(64.11921691894531, dtype=float32))\n",
      "('iteration', 19900, ':cross-entropy-loss', array(65.01276397705078, dtype=float32))\n",
      "('iteration', 20000, ':cross-entropy-loss', array(65.7929458618164, dtype=float32))\n",
      "('iteration', 20100, ':cross-entropy-loss', array(68.53535461425781, dtype=float32))\n",
      "('iteration', 20200, ':cross-entropy-loss', array(64.04862976074219, dtype=float32))\n",
      "('iteration', 20300, ':cross-entropy-loss', array(61.319393157958984, dtype=float32))\n",
      "('iteration', 20400, ':cross-entropy-loss', array(63.78238296508789, dtype=float32))\n",
      "('iteration', 20500, ':cross-entropy-loss', array(65.13278198242188, dtype=float32))\n",
      "('iteration', 20600, ':cross-entropy-loss', array(66.26957702636719, dtype=float32))\n",
      "('iteration', 20700, ':cross-entropy-loss', array(68.81229400634766, dtype=float32))\n",
      "('iteration', 20800, ':cross-entropy-loss', array(64.03363800048828, dtype=float32))\n",
      "('iteration', 20900, ':cross-entropy-loss', array(61.802833557128906, dtype=float32))\n",
      "('iteration', 21000, ':cross-entropy-loss', array(64.197509765625, dtype=float32))\n",
      "('iteration', 21100, ':cross-entropy-loss', array(65.49211883544922, dtype=float32))\n",
      "('iteration', 21200, ':cross-entropy-loss', array(66.29545593261719, dtype=float32))\n",
      "('iteration', 21300, ':cross-entropy-loss', array(69.21519470214844, dtype=float32))\n",
      "('iteration', 21400, ':cross-entropy-loss', array(64.0687484741211, dtype=float32))\n",
      "('iteration', 21500, ':cross-entropy-loss', array(62.219940185546875, dtype=float32))\n",
      "('iteration', 21600, ':cross-entropy-loss', array(64.69144439697266, dtype=float32))\n",
      "('iteration', 21700, ':cross-entropy-loss', array(65.84688568115234, dtype=float32))\n",
      "('iteration', 21800, ':cross-entropy-loss', array(66.53050994873047, dtype=float32))\n",
      "('iteration', 21900, ':cross-entropy-loss', array(69.43148040771484, dtype=float32))\n",
      "('iteration', 22000, ':cross-entropy-loss', array(64.4911117553711, dtype=float32))\n",
      "('iteration', 22100, ':cross-entropy-loss', array(62.6682243347168, dtype=float32))\n",
      "('iteration', 22200, ':cross-entropy-loss', array(65.04132080078125, dtype=float32))\n",
      "('iteration', 22300, ':cross-entropy-loss', array(66.03368377685547, dtype=float32))\n",
      "('iteration', 22400, ':cross-entropy-loss', array(66.42193603515625, dtype=float32))\n",
      "('iteration', 22500, ':cross-entropy-loss', array(69.8408432006836, dtype=float32))\n",
      "('iteration', 22600, ':cross-entropy-loss', array(64.67388153076172, dtype=float32))\n",
      "('iteration', 22700, ':cross-entropy-loss', array(62.795928955078125, dtype=float32))\n",
      "('iteration', 22800, ':cross-entropy-loss', array(65.46133422851562, dtype=float32))\n",
      "('iteration', 22900, ':cross-entropy-loss', array(66.20144653320312, dtype=float32))\n",
      "('iteration', 23000, ':cross-entropy-loss', array(67.49256896972656, dtype=float32))\n",
      "('iteration', 23100, ':cross-entropy-loss', array(70.4218978881836, dtype=float32))\n",
      "('iteration', 23200, ':cross-entropy-loss', array(65.32779693603516, dtype=float32))\n",
      "('iteration', 23300, ':cross-entropy-loss', array(62.95524597167969, dtype=float32))\n",
      "('iteration', 23400, ':cross-entropy-loss', array(66.30741119384766, dtype=float32))\n",
      "('iteration', 23500, ':cross-entropy-loss', array(66.0081558227539, dtype=float32))\n",
      "('iteration', 23600, ':cross-entropy-loss', array(68.01458740234375, dtype=float32))\n",
      "('iteration', 23700, ':cross-entropy-loss', array(70.70295715332031, dtype=float32))\n",
      "('iteration', 23800, ':cross-entropy-loss', array(65.3144760131836, dtype=float32))\n",
      "('iteration', 23900, ':cross-entropy-loss', array(63.54185104370117, dtype=float32))\n",
      "('iteration', 24000, ':cross-entropy-loss', array(66.22473907470703, dtype=float32))\n",
      "('iteration', 24100, ':cross-entropy-loss', array(66.39022064208984, dtype=float32))\n",
      "('iteration', 24200, ':cross-entropy-loss', array(68.32730865478516, dtype=float32))\n",
      "('iteration', 24300, ':cross-entropy-loss', array(70.80274200439453, dtype=float32))\n",
      "('iteration', 24400, ':cross-entropy-loss', array(65.59454345703125, dtype=float32))\n",
      "('iteration', 24500, ':cross-entropy-loss', array(63.5357666015625, dtype=float32))\n",
      "('iteration', 24600, ':cross-entropy-loss', array(66.9278793334961, dtype=float32))\n",
      "('iteration', 24700, ':cross-entropy-loss', array(66.72444915771484, dtype=float32))\n",
      "('iteration', 24800, ':cross-entropy-loss', array(68.54135131835938, dtype=float32))\n",
      "('iteration', 24900, ':cross-entropy-loss', array(70.67044067382812, dtype=float32))\n",
      "('iteration', 25000, ':cross-entropy-loss', array(65.3985824584961, dtype=float32))\n",
      "('iteration', 25100, ':cross-entropy-loss', array(63.909400939941406, dtype=float32))\n",
      "('iteration', 25200, ':cross-entropy-loss', array(67.2130355834961, dtype=float32))\n",
      "('iteration', 25300, ':cross-entropy-loss', array(66.5947494506836, dtype=float32))\n",
      "('iteration', 25400, ':cross-entropy-loss', array(68.41981506347656, dtype=float32))\n",
      "('iteration', 25500, ':cross-entropy-loss', array(70.84124755859375, dtype=float32))\n",
      "('iteration', 25600, ':cross-entropy-loss', array(65.50570678710938, dtype=float32))\n",
      "('iteration', 25700, ':cross-entropy-loss', array(64.09025573730469, dtype=float32))\n",
      "('iteration', 25800, ':cross-entropy-loss', array(66.9690170288086, dtype=float32))\n",
      "('iteration', 25900, ':cross-entropy-loss', array(67.31070709228516, dtype=float32))\n",
      "('iteration', 26000, ':cross-entropy-loss', array(68.33331298828125, dtype=float32))\n",
      "('iteration', 26100, ':cross-entropy-loss', array(71.06830596923828, dtype=float32))\n",
      "('iteration', 26200, ':cross-entropy-loss', array(65.5686264038086, dtype=float32))\n",
      "('iteration', 26300, ':cross-entropy-loss', array(63.81787109375, dtype=float32))\n",
      "('iteration', 26400, ':cross-entropy-loss', array(67.29032135009766, dtype=float32))\n",
      "('iteration', 26500, ':cross-entropy-loss', array(67.72882080078125, dtype=float32))\n",
      "('iteration', 26600, ':cross-entropy-loss', array(68.23033142089844, dtype=float32))\n",
      "('iteration', 26700, ':cross-entropy-loss', array(71.11958312988281, dtype=float32))\n",
      "('iteration', 26800, ':cross-entropy-loss', array(65.56843566894531, dtype=float32))\n",
      "('iteration', 26900, ':cross-entropy-loss', array(63.409210205078125, dtype=float32))\n",
      "('iteration', 27000, ':cross-entropy-loss', array(66.77340698242188, dtype=float32))\n",
      "('iteration', 27100, ':cross-entropy-loss', array(67.73908996582031, dtype=float32))\n",
      "('iteration', 27200, ':cross-entropy-loss', array(68.90019226074219, dtype=float32))\n",
      "('iteration', 27300, ':cross-entropy-loss', array(70.70503234863281, dtype=float32))\n",
      "('iteration', 27400, ':cross-entropy-loss', array(66.05742645263672, dtype=float32))\n",
      "('iteration', 27500, ':cross-entropy-loss', array(63.91064453125, dtype=float32))\n",
      "('iteration', 27600, ':cross-entropy-loss', array(67.12950897216797, dtype=float32))\n",
      "('iteration', 27700, ':cross-entropy-loss', array(67.67386627197266, dtype=float32))\n",
      "('iteration', 27800, ':cross-entropy-loss', array(69.16573333740234, dtype=float32))\n",
      "('iteration', 27900, ':cross-entropy-loss', array(70.85474395751953, dtype=float32))\n",
      "('iteration', 28000, ':cross-entropy-loss', array(65.9695053100586, dtype=float32))\n",
      "('iteration', 28100, ':cross-entropy-loss', array(64.12456512451172, dtype=float32))\n",
      "('iteration', 28200, ':cross-entropy-loss', array(66.69725036621094, dtype=float32))\n",
      "('iteration', 28300, ':cross-entropy-loss', array(68.1082992553711, dtype=float32))\n",
      "('iteration', 28400, ':cross-entropy-loss', array(69.28997802734375, dtype=float32))\n",
      "('iteration', 28500, ':cross-entropy-loss', array(71.15283966064453, dtype=float32))\n",
      "('iteration', 28600, ':cross-entropy-loss', array(65.81568908691406, dtype=float32))\n",
      "('iteration', 28700, ':cross-entropy-loss', array(64.15596008300781, dtype=float32))\n",
      "('iteration', 28800, ':cross-entropy-loss', array(67.04379272460938, dtype=float32))\n",
      "('iteration', 28900, ':cross-entropy-loss', array(68.08189392089844, dtype=float32))\n",
      "('iteration', 29000, ':cross-entropy-loss', array(69.07986450195312, dtype=float32))\n",
      "('iteration', 29100, ':cross-entropy-loss', array(71.5561294555664, dtype=float32))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('iteration', 29200, ':cross-entropy-loss', array(66.33019256591797, dtype=float32))\n",
      "('iteration', 29300, ':cross-entropy-loss', array(65.15589141845703, dtype=float32))\n",
      "('iteration', 29400, ':cross-entropy-loss', array(67.52595520019531, dtype=float32))\n",
      "('iteration', 29500, ':cross-entropy-loss', array(69.0166015625, dtype=float32))\n",
      "('iteration', 29600, ':cross-entropy-loss', array(70.10808563232422, dtype=float32))\n",
      "('iteration', 29700, ':cross-entropy-loss', array(72.64714050292969, dtype=float32))\n",
      "('iteration', 29800, ':cross-entropy-loss', array(67.29112243652344, dtype=float32))\n",
      "('iteration', 29900, ':cross-entropy-loss', array(65.35404205322266, dtype=float32))\n",
      "('iteration', 30000, ':cross-entropy-loss', array(67.9902114868164, dtype=float32))\n",
      "('iteration', 30100, ':cross-entropy-loss', array(69.01689147949219, dtype=float32))\n",
      "('iteration', 30200, ':cross-entropy-loss', array(69.9506607055664, dtype=float32))\n",
      "('iteration', 30300, ':cross-entropy-loss', array(72.990234375, dtype=float32))\n",
      "('iteration', 30400, ':cross-entropy-loss', array(67.3890380859375, dtype=float32))\n",
      "('iteration', 30500, ':cross-entropy-loss', array(65.19918060302734, dtype=float32))\n",
      "('iteration', 30600, ':cross-entropy-loss', array(67.92637634277344, dtype=float32))\n",
      "('iteration', 30700, ':cross-entropy-loss', array(69.27117919921875, dtype=float32))\n",
      "('iteration', 30800, ':cross-entropy-loss', array(70.04450988769531, dtype=float32))\n",
      "('iteration', 30900, ':cross-entropy-loss', array(72.86771392822266, dtype=float32))\n",
      "('iteration', 31000, ':cross-entropy-loss', array(67.51456451416016, dtype=float32))\n",
      "('iteration', 31100, ':cross-entropy-loss', array(64.7967529296875, dtype=float32))\n",
      "('iteration', 31200, ':cross-entropy-loss', array(68.0149154663086, dtype=float32))\n",
      "('iteration', 31300, ':cross-entropy-loss', array(69.60326385498047, dtype=float32))\n",
      "('iteration', 31400, ':cross-entropy-loss', array(70.32391357421875, dtype=float32))\n",
      "('iteration', 31500, ':cross-entropy-loss', array(73.3897933959961, dtype=float32))\n",
      "('iteration', 31600, ':cross-entropy-loss', array(67.63113403320312, dtype=float32))\n",
      "('iteration', 31700, ':cross-entropy-loss', array(65.24678802490234, dtype=float32))\n",
      "('iteration', 31800, ':cross-entropy-loss', array(67.97032928466797, dtype=float32))\n",
      "('iteration', 31900, ':cross-entropy-loss', array(69.50601959228516, dtype=float32))\n",
      "('iteration', 32000, ':cross-entropy-loss', array(70.32421112060547, dtype=float32))\n",
      "('iteration', 32100, ':cross-entropy-loss', array(73.74626922607422, dtype=float32))\n",
      "('iteration', 32200, ':cross-entropy-loss', array(67.84803009033203, dtype=float32))\n",
      "('iteration', 32300, ':cross-entropy-loss', array(65.59783935546875, dtype=float32))\n",
      "('iteration', 32400, ':cross-entropy-loss', array(68.24066162109375, dtype=float32))\n",
      "('iteration', 32500, ':cross-entropy-loss', array(69.68953704833984, dtype=float32))\n",
      "('iteration', 32600, ':cross-entropy-loss', array(70.81896209716797, dtype=float32))\n",
      "('iteration', 32700, ':cross-entropy-loss', array(73.79320526123047, dtype=float32))\n",
      "('iteration', 32800, ':cross-entropy-loss', array(67.85677337646484, dtype=float32))\n",
      "('iteration', 32900, ':cross-entropy-loss', array(65.61368560791016, dtype=float32))\n",
      "('iteration', 33000, ':cross-entropy-loss', array(68.38378143310547, dtype=float32))\n",
      "('iteration', 33100, ':cross-entropy-loss', array(69.90788269042969, dtype=float32))\n",
      "('iteration', 33200, ':cross-entropy-loss', array(71.18445587158203, dtype=float32))\n",
      "('iteration', 33300, ':cross-entropy-loss', array(73.91476440429688, dtype=float32))\n",
      "('iteration', 33400, ':cross-entropy-loss', array(68.38482666015625, dtype=float32))\n",
      "('iteration', 33500, ':cross-entropy-loss', array(65.6861343383789, dtype=float32))\n",
      "('iteration', 33600, ':cross-entropy-loss', array(68.75540924072266, dtype=float32))\n",
      "('iteration', 33700, ':cross-entropy-loss', array(70.20602416992188, dtype=float32))\n",
      "('iteration', 33800, ':cross-entropy-loss', array(71.23268127441406, dtype=float32))\n",
      "('iteration', 33900, ':cross-entropy-loss', array(74.81116485595703, dtype=float32))\n",
      "('iteration', 34000, ':cross-entropy-loss', array(68.72811889648438, dtype=float32))\n",
      "('iteration', 34100, ':cross-entropy-loss', array(66.17105102539062, dtype=float32))\n",
      "('iteration', 34200, ':cross-entropy-loss', array(68.91403198242188, dtype=float32))\n",
      "('iteration', 34300, ':cross-entropy-loss', array(69.97145080566406, dtype=float32))\n",
      "('iteration', 34400, ':cross-entropy-loss', array(71.63982391357422, dtype=float32))\n",
      "('iteration', 34500, ':cross-entropy-loss', array(74.92744445800781, dtype=float32))\n",
      "('iteration', 34600, ':cross-entropy-loss', array(69.1950454711914, dtype=float32))\n",
      "('iteration', 34700, ':cross-entropy-loss', array(66.453369140625, dtype=float32))\n",
      "('iteration', 34800, ':cross-entropy-loss', array(69.8639907836914, dtype=float32))\n",
      "('iteration', 34900, ':cross-entropy-loss', array(71.7382583618164, dtype=float32))\n",
      "('iteration', 35000, ':cross-entropy-loss', array(72.23086547851562, dtype=float32))\n",
      "('iteration', 35100, ':cross-entropy-loss', array(76.17438507080078, dtype=float32))\n",
      "('iteration', 35200, ':cross-entropy-loss', array(69.96536254882812, dtype=float32))\n",
      "('iteration', 35300, ':cross-entropy-loss', array(66.5099105834961, dtype=float32))\n",
      "('iteration', 35400, ':cross-entropy-loss', array(70.87689208984375, dtype=float32))\n",
      "('iteration', 35500, ':cross-entropy-loss', array(71.70238494873047, dtype=float32))\n",
      "('iteration', 35600, ':cross-entropy-loss', array(72.45833587646484, dtype=float32))\n",
      "('iteration', 35700, ':cross-entropy-loss', array(75.91490936279297, dtype=float32))\n",
      "('iteration', 35800, ':cross-entropy-loss', array(69.6169662475586, dtype=float32))\n",
      "('iteration', 35900, ':cross-entropy-loss', array(66.6960678100586, dtype=float32))\n",
      "('iteration', 36000, ':cross-entropy-loss', array(71.19227600097656, dtype=float32))\n",
      "('iteration', 36100, ':cross-entropy-loss', array(71.6521987915039, dtype=float32))\n",
      "('iteration', 36200, ':cross-entropy-loss', array(72.8400650024414, dtype=float32))\n",
      "('iteration', 36300, ':cross-entropy-loss', array(76.38909912109375, dtype=float32))\n",
      "('iteration', 36400, ':cross-entropy-loss', array(69.88993835449219, dtype=float32))\n",
      "('iteration', 36500, ':cross-entropy-loss', array(66.68958282470703, dtype=float32))\n",
      "('iteration', 36600, ':cross-entropy-loss', array(71.03221893310547, dtype=float32))\n",
      "('iteration', 36700, ':cross-entropy-loss', array(71.78329467773438, dtype=float32))\n",
      "('iteration', 36800, ':cross-entropy-loss', array(73.42840576171875, dtype=float32))\n",
      "('iteration', 36900, ':cross-entropy-loss', array(76.59252166748047, dtype=float32))\n",
      "('iteration', 37000, ':cross-entropy-loss', array(70.52201843261719, dtype=float32))\n",
      "('iteration', 37100, ':cross-entropy-loss', array(67.13859558105469, dtype=float32))\n",
      "('iteration', 37200, ':cross-entropy-loss', array(71.30272674560547, dtype=float32))\n",
      "('iteration', 37300, ':cross-entropy-loss', array(72.11544036865234, dtype=float32))\n",
      "('iteration', 37400, ':cross-entropy-loss', array(73.48310852050781, dtype=float32))\n",
      "('iteration', 37500, ':cross-entropy-loss', array(76.6174087524414, dtype=float32))\n",
      "('iteration', 37600, ':cross-entropy-loss', array(70.65049743652344, dtype=float32))\n",
      "('iteration', 37700, ':cross-entropy-loss', array(67.02886962890625, dtype=float32))\n",
      "('iteration', 37800, ':cross-entropy-loss', array(72.17292022705078, dtype=float32))\n",
      "('iteration', 37900, ':cross-entropy-loss', array(72.29137420654297, dtype=float32))\n",
      "('iteration', 38000, ':cross-entropy-loss', array(73.16109466552734, dtype=float32))\n",
      "('iteration', 38100, ':cross-entropy-loss', array(76.96615600585938, dtype=float32))\n",
      "('iteration', 38200, ':cross-entropy-loss', array(70.7264404296875, dtype=float32))\n",
      "('iteration', 38300, ':cross-entropy-loss', array(67.1092300415039, dtype=float32))\n",
      "('iteration', 38400, ':cross-entropy-loss', array(72.2400894165039, dtype=float32))\n",
      "('iteration', 38500, ':cross-entropy-loss', array(72.62983703613281, dtype=float32))\n",
      "('iteration', 38600, ':cross-entropy-loss', array(73.4517593383789, dtype=float32))\n",
      "('iteration', 38700, ':cross-entropy-loss', array(77.1421890258789, dtype=float32))\n",
      "('iteration', 38800, ':cross-entropy-loss', array(70.64682006835938, dtype=float32))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('iteration', 38900, ':cross-entropy-loss', array(67.11483001708984, dtype=float32))\n",
      "('iteration', 39000, ':cross-entropy-loss', array(72.0860824584961, dtype=float32))\n",
      "('iteration', 39100, ':cross-entropy-loss', array(72.71981811523438, dtype=float32))\n",
      "('iteration', 39200, ':cross-entropy-loss', array(74.176025390625, dtype=float32))\n",
      "('iteration', 39300, ':cross-entropy-loss', array(76.81652069091797, dtype=float32))\n",
      "('iteration', 39400, ':cross-entropy-loss', array(71.09933471679688, dtype=float32))\n",
      "('iteration', 39500, ':cross-entropy-loss', array(67.45205688476562, dtype=float32))\n",
      "('iteration', 39600, ':cross-entropy-loss', array(72.31353759765625, dtype=float32))\n",
      "('iteration', 39700, ':cross-entropy-loss', array(72.70954895019531, dtype=float32))\n",
      "('iteration', 39800, ':cross-entropy-loss', array(74.73269653320312, dtype=float32))\n",
      "('iteration', 39900, ':cross-entropy-loss', array(76.9212417602539, dtype=float32))\n",
      "('iteration', 40000, ':cross-entropy-loss', array(71.33551025390625, dtype=float32))\n",
      "('iteration', 40100, ':cross-entropy-loss', array(67.7189712524414, dtype=float32))\n",
      "('iteration', 40200, ':cross-entropy-loss', array(72.6503677368164, dtype=float32))\n",
      "('iteration', 40300, ':cross-entropy-loss', array(73.104248046875, dtype=float32))\n",
      "('iteration', 40400, ':cross-entropy-loss', array(74.92608642578125, dtype=float32))\n",
      "('iteration', 40500, ':cross-entropy-loss', array(77.55374145507812, dtype=float32))\n",
      "('iteration', 40600, ':cross-entropy-loss', array(71.97665405273438, dtype=float32))\n",
      "('iteration', 40700, ':cross-entropy-loss', array(69.0954360961914, dtype=float32))\n",
      "('iteration', 40800, ':cross-entropy-loss', array(72.61351776123047, dtype=float32))\n",
      "('iteration', 40900, ':cross-entropy-loss', array(73.87312316894531, dtype=float32))\n",
      "('iteration', 41000, ':cross-entropy-loss', array(75.84347534179688, dtype=float32))\n",
      "('iteration', 41100, ':cross-entropy-loss', array(77.95852661132812, dtype=float32))\n",
      "('iteration', 41200, ':cross-entropy-loss', array(71.99870300292969, dtype=float32))\n",
      "('iteration', 41300, ':cross-entropy-loss', array(68.8971939086914, dtype=float32))\n",
      "('iteration', 41400, ':cross-entropy-loss', array(73.02654266357422, dtype=float32))\n",
      "('iteration', 41500, ':cross-entropy-loss', array(74.5563735961914, dtype=float32))\n",
      "('iteration', 41600, ':cross-entropy-loss', array(75.7680892944336, dtype=float32))\n",
      "('iteration', 41700, ':cross-entropy-loss', array(78.09871673583984, dtype=float32))\n",
      "('iteration', 41800, ':cross-entropy-loss', array(72.34490966796875, dtype=float32))\n",
      "('iteration', 41900, ':cross-entropy-loss', array(69.56072998046875, dtype=float32))\n",
      "('iteration', 42000, ':cross-entropy-loss', array(72.35606384277344, dtype=float32))\n",
      "('iteration', 42100, ':cross-entropy-loss', array(74.41525268554688, dtype=float32))\n",
      "('iteration', 42200, ':cross-entropy-loss', array(75.81886291503906, dtype=float32))\n",
      "('iteration', 42300, ':cross-entropy-loss', array(77.87687683105469, dtype=float32))\n",
      "('iteration', 42400, ':cross-entropy-loss', array(72.45401000976562, dtype=float32))\n",
      "('iteration', 42500, ':cross-entropy-loss', array(69.82372283935547, dtype=float32))\n",
      "('iteration', 42600, ':cross-entropy-loss', array(72.72575378417969, dtype=float32))\n",
      "('iteration', 42700, ':cross-entropy-loss', array(75.05284881591797, dtype=float32))\n",
      "('iteration', 42800, ':cross-entropy-loss', array(76.18367004394531, dtype=float32))\n",
      "('iteration', 42900, ':cross-entropy-loss', array(78.31513977050781, dtype=float32))\n",
      "('iteration', 43000, ':cross-entropy-loss', array(72.2883529663086, dtype=float32))\n",
      "('iteration', 43100, ':cross-entropy-loss', array(69.58636474609375, dtype=float32))\n",
      "('iteration', 43200, ':cross-entropy-loss', array(73.0554428100586, dtype=float32))\n",
      "('iteration', 43300, ':cross-entropy-loss', array(75.04175567626953, dtype=float32))\n",
      "('iteration', 43400, ':cross-entropy-loss', array(75.5730209350586, dtype=float32))\n",
      "('iteration', 43500, ':cross-entropy-loss', array(79.3382797241211, dtype=float32))\n",
      "('iteration', 43600, ':cross-entropy-loss', array(72.68292236328125, dtype=float32))\n",
      "('iteration', 43700, ':cross-entropy-loss', array(69.18934631347656, dtype=float32))\n",
      "('iteration', 43800, ':cross-entropy-loss', array(72.68065643310547, dtype=float32))\n",
      "('iteration', 43900, ':cross-entropy-loss', array(75.52662658691406, dtype=float32))\n",
      "('iteration', 44000, ':cross-entropy-loss', array(75.03564453125, dtype=float32))\n",
      "('iteration', 44100, ':cross-entropy-loss', array(79.08055877685547, dtype=float32))\n",
      "('iteration', 44200, ':cross-entropy-loss', array(72.77584838867188, dtype=float32))\n",
      "('iteration', 44300, ':cross-entropy-loss', array(69.35296630859375, dtype=float32))\n",
      "('iteration', 44400, ':cross-entropy-loss', array(72.35936737060547, dtype=float32))\n",
      "('iteration', 44500, ':cross-entropy-loss', array(75.5412368774414, dtype=float32))\n",
      "('iteration', 44600, ':cross-entropy-loss', array(75.34896850585938, dtype=float32))\n",
      "('iteration', 44700, ':cross-entropy-loss', array(79.06221771240234, dtype=float32))\n",
      "('iteration', 44800, ':cross-entropy-loss', array(72.32852172851562, dtype=float32))\n",
      "('iteration', 44900, ':cross-entropy-loss', array(69.3328857421875, dtype=float32))\n",
      "('iteration', 45000, ':cross-entropy-loss', array(72.26873779296875, dtype=float32))\n",
      "('iteration', 45100, ':cross-entropy-loss', array(75.88322448730469, dtype=float32))\n",
      "('iteration', 45200, ':cross-entropy-loss', array(75.26632690429688, dtype=float32))\n",
      "('iteration', 45300, ':cross-entropy-loss', array(79.3812026977539, dtype=float32))\n",
      "('iteration', 45400, ':cross-entropy-loss', array(72.08287811279297, dtype=float32))\n",
      "('iteration', 45500, ':cross-entropy-loss', array(69.12447357177734, dtype=float32))\n",
      "('iteration', 45600, ':cross-entropy-loss', array(72.2414779663086, dtype=float32))\n",
      "('iteration', 45700, ':cross-entropy-loss', array(75.02210235595703, dtype=float32))\n",
      "('iteration', 45800, ':cross-entropy-loss', array(75.22761535644531, dtype=float32))\n",
      "('iteration', 45900, ':cross-entropy-loss', array(79.06413269042969, dtype=float32))\n",
      "('iteration', 46000, ':cross-entropy-loss', array(72.23495483398438, dtype=float32))\n",
      "('iteration', 46100, ':cross-entropy-loss', array(69.33939361572266, dtype=float32))\n",
      "('iteration', 46200, ':cross-entropy-loss', array(72.54098510742188, dtype=float32))\n",
      "('iteration', 46300, ':cross-entropy-loss', array(75.36626434326172, dtype=float32))\n",
      "('iteration', 46400, ':cross-entropy-loss', array(75.25563049316406, dtype=float32))\n",
      "('iteration', 46500, ':cross-entropy-loss', array(79.6480484008789, dtype=float32))\n",
      "('iteration', 46600, ':cross-entropy-loss', array(72.41349029541016, dtype=float32))\n",
      "('iteration', 46700, ':cross-entropy-loss', array(69.13518524169922, dtype=float32))\n",
      "('iteration', 46800, ':cross-entropy-loss', array(72.555419921875, dtype=float32))\n",
      "('iteration', 46900, ':cross-entropy-loss', array(75.5836410522461, dtype=float32))\n",
      "('iteration', 47000, ':cross-entropy-loss', array(75.54812622070312, dtype=float32))\n",
      "('iteration', 47100, ':cross-entropy-loss', array(79.51431274414062, dtype=float32))\n",
      "('iteration', 47200, ':cross-entropy-loss', array(72.5434341430664, dtype=float32))\n",
      "('iteration', 47300, ':cross-entropy-loss', array(70.74302673339844, dtype=float32))\n",
      "('iteration', 47400, ':cross-entropy-loss', array(73.12720489501953, dtype=float32))\n",
      "('iteration', 47500, ':cross-entropy-loss', array(76.03135681152344, dtype=float32))\n",
      "('iteration', 47600, ':cross-entropy-loss', array(75.83583068847656, dtype=float32))\n",
      "('iteration', 47700, ':cross-entropy-loss', array(79.81275939941406, dtype=float32))\n",
      "('iteration', 47800, ':cross-entropy-loss', array(72.4760513305664, dtype=float32))\n",
      "('iteration', 47900, ':cross-entropy-loss', array(69.91078186035156, dtype=float32))\n",
      "('iteration', 48000, ':cross-entropy-loss', array(73.07089233398438, dtype=float32))\n",
      "('iteration', 48100, ':cross-entropy-loss', array(75.35404205322266, dtype=float32))\n",
      "('iteration', 48200, ':cross-entropy-loss', array(75.6845932006836, dtype=float32))\n",
      "('iteration', 48300, ':cross-entropy-loss', array(79.92269897460938, dtype=float32))\n",
      "('iteration', 48400, ':cross-entropy-loss', array(72.2166976928711, dtype=float32))\n",
      "('iteration', 48500, ':cross-entropy-loss', array(69.21107482910156, dtype=float32))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('iteration', 48600, ':cross-entropy-loss', array(73.05695343017578, dtype=float32))\n",
      "('iteration', 48700, ':cross-entropy-loss', array(74.69354248046875, dtype=float32))\n",
      "('iteration', 48800, ':cross-entropy-loss', array(75.33463287353516, dtype=float32))\n",
      "('iteration', 48900, ':cross-entropy-loss', array(79.4066390991211, dtype=float32))\n",
      "('iteration', 49000, ':cross-entropy-loss', array(71.9058837890625, dtype=float32))\n",
      "('iteration', 49100, ':cross-entropy-loss', array(69.20668029785156, dtype=float32))\n",
      "('iteration', 49200, ':cross-entropy-loss', array(72.8581314086914, dtype=float32))\n",
      "('iteration', 49300, ':cross-entropy-loss', array(74.98412322998047, dtype=float32))\n",
      "('iteration', 49400, ':cross-entropy-loss', array(75.09954833984375, dtype=float32))\n",
      "('iteration', 49500, ':cross-entropy-loss', array(79.64055633544922, dtype=float32))\n",
      "('iteration', 49600, ':cross-entropy-loss', array(71.95227813720703, dtype=float32))\n",
      "('iteration', 49700, ':cross-entropy-loss', array(69.2331771850586, dtype=float32))\n",
      "('iteration', 49800, ':cross-entropy-loss', array(73.17959594726562, dtype=float32))\n",
      "('iteration', 49900, ':cross-entropy-loss', array(74.20365142822266, dtype=float32))\n",
      "('iteration', 50000, ':cross-entropy-loss', array(75.17776489257812, dtype=float32))\n",
      "('iteration', 50100, ':cross-entropy-loss', array(79.5213394165039, dtype=float32))\n",
      "('iteration', 50200, ':cross-entropy-loss', array(72.04158020019531, dtype=float32))\n",
      "('iteration', 50300, ':cross-entropy-loss', array(69.3228530883789, dtype=float32))\n",
      "('iteration', 50400, ':cross-entropy-loss', array(72.77364349365234, dtype=float32))\n",
      "('iteration', 50500, ':cross-entropy-loss', array(74.10819244384766, dtype=float32))\n",
      "('iteration', 50600, ':cross-entropy-loss', array(75.40597534179688, dtype=float32))\n",
      "('iteration', 50700, ':cross-entropy-loss', array(79.19727325439453, dtype=float32))\n",
      "('iteration', 50800, ':cross-entropy-loss', array(72.28089904785156, dtype=float32))\n",
      "('iteration', 50900, ':cross-entropy-loss', array(69.85356903076172, dtype=float32))\n",
      "('iteration', 51000, ':cross-entropy-loss', array(73.21867370605469, dtype=float32))\n",
      "('iteration', 51100, ':cross-entropy-loss', array(75.15435791015625, dtype=float32))\n",
      "('iteration', 51200, ':cross-entropy-loss', array(75.92052459716797, dtype=float32))\n",
      "('iteration', 51300, ':cross-entropy-loss', array(79.24107360839844, dtype=float32))\n",
      "('iteration', 51400, ':cross-entropy-loss', array(72.4499282836914, dtype=float32))\n",
      "('iteration', 51500, ':cross-entropy-loss', array(70.65618133544922, dtype=float32))\n",
      "('iteration', 51600, ':cross-entropy-loss', array(73.58580017089844, dtype=float32))\n",
      "('iteration', 51700, ':cross-entropy-loss', array(74.96405029296875, dtype=float32))\n",
      "('iteration', 51800, ':cross-entropy-loss', array(75.91128540039062, dtype=float32))\n",
      "('iteration', 51900, ':cross-entropy-loss', array(79.5100326538086, dtype=float32))\n",
      "('iteration', 52000, ':cross-entropy-loss', array(72.45826721191406, dtype=float32))\n",
      "('iteration', 52100, ':cross-entropy-loss', array(71.12823486328125, dtype=float32))\n",
      "('iteration', 52200, ':cross-entropy-loss', array(73.39839172363281, dtype=float32))\n",
      "('iteration', 52300, ':cross-entropy-loss', array(75.0423583984375, dtype=float32))\n",
      "('iteration', 52400, ':cross-entropy-loss', array(75.86444091796875, dtype=float32))\n",
      "('iteration', 52500, ':cross-entropy-loss', array(80.47602081298828, dtype=float32))\n",
      "('iteration', 52600, ':cross-entropy-loss', array(73.18938446044922, dtype=float32))\n",
      "('iteration', 52700, ':cross-entropy-loss', array(71.83949279785156, dtype=float32))\n",
      "('iteration', 52800, ':cross-entropy-loss', array(73.78983306884766, dtype=float32))\n",
      "('iteration', 52900, ':cross-entropy-loss', array(75.46879577636719, dtype=float32))\n",
      "('iteration', 53000, ':cross-entropy-loss', array(75.92455291748047, dtype=float32))\n",
      "('iteration', 53100, ':cross-entropy-loss', array(81.10066223144531, dtype=float32))\n",
      "('iteration', 53200, ':cross-entropy-loss', array(73.14303588867188, dtype=float32))\n",
      "('iteration', 53300, ':cross-entropy-loss', array(71.9834976196289, dtype=float32))\n",
      "('iteration', 53400, ':cross-entropy-loss', array(74.11503601074219, dtype=float32))\n",
      "('iteration', 53500, ':cross-entropy-loss', array(76.29299926757812, dtype=float32))\n",
      "('iteration', 53600, ':cross-entropy-loss', array(76.73551940917969, dtype=float32))\n",
      "('iteration', 53700, ':cross-entropy-loss', array(81.02717590332031, dtype=float32))\n",
      "('iteration', 53800, ':cross-entropy-loss', array(73.78905487060547, dtype=float32))\n",
      "('iteration', 53900, ':cross-entropy-loss', array(72.43360900878906, dtype=float32))\n",
      "('iteration', 54000, ':cross-entropy-loss', array(76.09184265136719, dtype=float32))\n",
      "('iteration', 54100, ':cross-entropy-loss', array(77.04907989501953, dtype=float32))\n",
      "('iteration', 54200, ':cross-entropy-loss', array(77.61135864257812, dtype=float32))\n",
      "('iteration', 54300, ':cross-entropy-loss', array(80.8628921508789, dtype=float32))\n",
      "('iteration', 54400, ':cross-entropy-loss', array(74.302001953125, dtype=float32))\n",
      "('iteration', 54500, ':cross-entropy-loss', array(72.7918930053711, dtype=float32))\n",
      "('iteration', 54600, ':cross-entropy-loss', array(76.0271987915039, dtype=float32))\n",
      "('iteration', 54700, ':cross-entropy-loss', array(76.93330383300781, dtype=float32))\n",
      "('iteration', 54800, ':cross-entropy-loss', array(77.78821563720703, dtype=float32))\n",
      "('iteration', 54900, ':cross-entropy-loss', array(81.8856201171875, dtype=float32))\n",
      "('iteration', 55000, ':cross-entropy-loss', array(74.53958892822266, dtype=float32))\n",
      "('iteration', 55100, ':cross-entropy-loss', array(72.67791748046875, dtype=float32))\n",
      "('iteration', 55200, ':cross-entropy-loss', array(75.81875610351562, dtype=float32))\n",
      "('iteration', 55300, ':cross-entropy-loss', array(76.82638549804688, dtype=float32))\n",
      "('iteration', 55400, ':cross-entropy-loss', array(78.21934509277344, dtype=float32))\n",
      "('iteration', 55500, ':cross-entropy-loss', array(82.25697326660156, dtype=float32))\n",
      "('iteration', 55600, ':cross-entropy-loss', array(74.04932403564453, dtype=float32))\n",
      "('iteration', 55700, ':cross-entropy-loss', array(73.147705078125, dtype=float32))\n",
      "('iteration', 55800, ':cross-entropy-loss', array(75.9338607788086, dtype=float32))\n",
      "('iteration', 55900, ':cross-entropy-loss', array(77.23783874511719, dtype=float32))\n",
      "('iteration', 56000, ':cross-entropy-loss', array(77.71269989013672, dtype=float32))\n",
      "('iteration', 56100, ':cross-entropy-loss', array(82.93696594238281, dtype=float32))\n",
      "('iteration', 56200, ':cross-entropy-loss', array(74.58729553222656, dtype=float32))\n",
      "('iteration', 56300, ':cross-entropy-loss', array(72.794189453125, dtype=float32))\n",
      "('iteration', 56400, ':cross-entropy-loss', array(76.02550506591797, dtype=float32))\n",
      "('iteration', 56500, ':cross-entropy-loss', array(77.27816772460938, dtype=float32))\n",
      "('iteration', 56600, ':cross-entropy-loss', array(78.06324005126953, dtype=float32))\n",
      "('iteration', 56700, ':cross-entropy-loss', array(83.20015716552734, dtype=float32))\n",
      "('iteration', 56800, ':cross-entropy-loss', array(74.90523529052734, dtype=float32))\n",
      "('iteration', 56900, ':cross-entropy-loss', array(72.73945617675781, dtype=float32))\n",
      "('iteration', 57000, ':cross-entropy-loss', array(75.6607666015625, dtype=float32))\n",
      "('iteration', 57100, ':cross-entropy-loss', array(77.6541519165039, dtype=float32))\n",
      "('iteration', 57200, ':cross-entropy-loss', array(77.4377212524414, dtype=float32))\n",
      "('iteration', 57300, ':cross-entropy-loss', array(83.0085678100586, dtype=float32))\n",
      "('iteration', 57400, ':cross-entropy-loss', array(74.93948364257812, dtype=float32))\n",
      "('iteration', 57500, ':cross-entropy-loss', array(73.21216583251953, dtype=float32))\n",
      "('iteration', 57600, ':cross-entropy-loss', array(76.32801818847656, dtype=float32))\n",
      "('iteration', 57700, ':cross-entropy-loss', array(77.60718536376953, dtype=float32))\n",
      "('iteration', 57800, ':cross-entropy-loss', array(77.99333953857422, dtype=float32))\n",
      "('iteration', 57900, ':cross-entropy-loss', array(82.86177825927734, dtype=float32))\n",
      "('iteration', 58000, ':cross-entropy-loss', array(75.3124008178711, dtype=float32))\n",
      "('iteration', 58100, ':cross-entropy-loss', array(73.27372741699219, dtype=float32))\n",
      "('iteration', 58200, ':cross-entropy-loss', array(76.83914184570312, dtype=float32))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('iteration', 58300, ':cross-entropy-loss', array(77.07289123535156, dtype=float32))\n",
      "('iteration', 58400, ':cross-entropy-loss', array(78.13334655761719, dtype=float32))\n",
      "('iteration', 58500, ':cross-entropy-loss', array(83.74488067626953, dtype=float32))\n",
      "('iteration', 58600, ':cross-entropy-loss', array(75.88180541992188, dtype=float32))\n",
      "('iteration', 58700, ':cross-entropy-loss', array(73.3929214477539, dtype=float32))\n",
      "('iteration', 58800, ':cross-entropy-loss', array(77.63650512695312, dtype=float32))\n",
      "('iteration', 58900, ':cross-entropy-loss', array(77.39779663085938, dtype=float32))\n",
      "('iteration', 59000, ':cross-entropy-loss', array(78.18351745605469, dtype=float32))\n",
      "('iteration', 59100, ':cross-entropy-loss', array(83.43802642822266, dtype=float32))\n",
      "('iteration', 59200, ':cross-entropy-loss', array(75.57208251953125, dtype=float32))\n",
      "('iteration', 59300, ':cross-entropy-loss', array(73.57415008544922, dtype=float32))\n",
      "('iteration', 59400, ':cross-entropy-loss', array(77.0608139038086, dtype=float32))\n",
      "('iteration', 59500, ':cross-entropy-loss', array(77.50554656982422, dtype=float32))\n",
      "('iteration', 59600, ':cross-entropy-loss', array(79.00891876220703, dtype=float32))\n",
      "('iteration', 59700, ':cross-entropy-loss', array(83.41793823242188, dtype=float32))\n",
      "('iteration', 59800, ':cross-entropy-loss', array(75.86214447021484, dtype=float32))\n",
      "('iteration', 59900, ':cross-entropy-loss', array(74.24818420410156, dtype=float32))\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.insert(0,'/home/csunix/schtmt/NewFolder/caffe_Sep/python')\n",
    "sys.path.insert(0,'/home/csunix/schtmt/NewFolder/caffe_Sep/examples/mnist_wta_autoencoder')\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import caffe\n",
    "#% matplotlib inline#THIS LINE HAS ERROR -> USE THE FOLLOWING 2 LINES\n",
    "# from IPython import get_ipython\n",
    "# get_ipython().run_line_magic('matplotlib', 'inline')\n",
    "caffe.set_device(0)\n",
    "caffe.set_mode_gpu()\n",
    "solver = caffe.SGDSolver('mnist_wta_cross_entropy_solver.prototxt')\n",
    "\n",
    "#fine-tune the weight of layers having a same names, train from scratch for other layers\n",
    "# solver.net.copy_from('/usr/not-backed-up/1_convlstm/mnist_wta_cross_entropy_loss_sep2017_1/mnist_GlobalWta_iter_9600.caffemodel')\n",
    "niter = 60000\n",
    "train_loss = np.zeros(niter)\n",
    "\n",
    "for i in range(niter):\n",
    "    solver.step(1)\n",
    "    #print solver.net.blobs.keys()\n",
    "    loss = solver.net.blobs['cross_entropy_loss'].data\n",
    "    if (i%100 == 0):\n",
    "        print('iteration', i, ':cross-entropy-loss', loss) \n",
    "    train_loss[i] = loss\n",
    "    np.save('train_cross_entropy_loss_Sep2017',train_loss)\n",
    "    #np.save('train_loss', train_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
